package log

import java.time.LocalDateTime
import java.time.format.DateTimeFormatter
import java.util.Locale
import java.util.regex.{Matcher, Pattern}

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object LogAnalyse {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("CdnLogAnalyse").setMaster("local[*]")
    val sc = new SparkContext(conf)
    sc.setLogLevel("warn")

    val cdnLog = sc.textFile("data/cdn.txt")
    val rdd: RDD[(String, String, String)] = cdnLog.map { line =>
      val fields: Array[String] = line.split("\\s+")
      (fields(0) ,fields(3), fields(6))
    }
    //rdd.foreach(println)

    //独立ip
    val ipMap=rdd.map( key=> (key._1,1)).reduceByKey(_+_)
    //ipMap.foreach(println)

      println("独立ip数："+ipMap.count())


   /* println(rdd.filter {
      key => key._3.endsWith(".mp4")
    }.map {
      key => ((key._1, key._3), 1)
    }.count())*/

    //视频独立ip
    val video: RDD[(String, Int)] = rdd.filter {
      key => key._3.endsWith(".mp4")
    }.map {
      key => ((key._1, key._3), 1)
    }.reduceByKey(_ + _).map {
      case ((_, video), _) => (video, 1)
    }.reduceByKey(_ + _)

    println("视频独立IP数:")
    video.foreach(x => println(x._1+" "+x._2))



    //videoIp.foreach(println)
    //每小时流量
    println("每小时流量:")
    val timeIp: collection.Map[String, Long] = rdd.map(key => (parseDate(key._2.substring(1)), 1))
      .countByKey()

    timeIp.foreach(key => println(key._1 + " " +key._2))

    sc.stop()



  }


  def parseDate(dateStr: String):String = {
    try {
      val formatter: DateTimeFormatter = DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss", Locale.ENGLISH)
      val localDateTime = LocalDateTime.parse(dateStr, formatter)
      val str = localDateTime.format(DateTimeFormatter.ofPattern("yyyyMMddHH"))
      // 假设小于2000年和大于当前年的都是无效日期
      if (str < "2000000000" || str > LocalDateTime.now().getYear.toString()) return null
      str
    } catch {
      case _: Exception =>
        println(s"日期解析失败：$dateStr")
        null
    }
  }
}
